package org.eduapp.trains.crawler;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;
import org.eduapp.trains.ServiceException;
import org.eduapp.trains.model.train.ModelTrain;
import org.eduapp.trains.model.train.ModelTrainSchedule;
import org.eduapp.trains.service.train.TrainService;
import org.eduapp.trains.utils.UtilsString;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;

/**
 * 数据源采集: http://www.tielu.org/
 * 
 * @author JeccyZhao
 *
 */
public class TrainDataCrawler extends WebCrawler {
	
	private final static Logger logger = Logger.getLogger(TrainDataCrawler.class);
	
	/*
	 * should be removed after web integration.
	 * only for data crawling at first time.
	 */
	private ApplicationContext applicationContext = new ClassPathXmlApplicationContext(new String[]{"spring/application*.xml"});
	
	/*
	 * service for train model. 
	 * fetching and saving or updating
	 */
	private TrainService trainService = (TrainService)applicationContext.getBean("trainService");
	
	/*
	 * counter to record 
	 * the number of train saved
	 */
	private int count;
	
	/*
	 * train data read from database.
	 * avoid the useless request and parse
	 */
	private Set<String> dataFromDB;
	
	/*
	 * pages can not be accessed.
	 * might be caused by exception or something else.
	 */
	private List<String> errorPages;
	
	/*
	 * pattern to filter meta in the page
	 * 
	 */
	private Pattern filters = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"
			+ "|png|tiff?|mid|mp2|mp3|mp4" + "|wav|avi|mov|mpeg|ram|m4v|pdf"
			+ "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
	
	public TrainDataCrawler () {
		init();
	}
	
	/*
	 * fetch train data from database
	 */
	private void init() {
		try {
			List<ModelTrain> data = trainService.getAll();
			if (data != null) {
				if (dataFromDB == null) {
					dataFromDB = new HashSet<String>();
				}
				for (ModelTrain model : data) {
					dataFromDB.add(model.getTrainNumber().toLowerCase());
				}
			}
		} catch (ServiceException e) {
			e.printStackTrace();
			logger.error("fetching train data failed, " + e.getMessage());
		}
	}
	
	public boolean shouldVisit(WebURL url) {
		if (url != null && url.getURL() != null) {
			String href = url.getURL().toLowerCase();
			
			if (filters.matcher(href).matches()) {
				return false;
			}
			
			if (isTrainListPage(href)) {
				return true;
			}
			
			if (isTrainPage(href)) {
				String trainNumber = translateTrainNo(href);
				if (trainNumber != null && !dataFromDB.contains(trainNumber)) {
					return true;
				} else {
					logger.info("Data hit in DB " + " > " + trainNumber);
					System.out.println("Data hit in DB " + " > " + trainNumber);
					return false;
				}
			}
		}
		return false;
	}
	
	public void visit(Page page) {
		
		String url = page.getWebURL().getURL();
		
		/*
		 * should be the train page
		 */
		if (isTrainPage(url)) {
			ModelTrain modelTrain = getTrainInfo(page.getHTML(), url);
			if (modelTrain != null) {
				count++;
				try {
					if (trainService.getByTrainNo(modelTrain.getTrainNumber()) == null) {
						trainService.saveOrUpdate(modelTrain);
						dataFromDB.add(modelTrain.getTrainNumber());
						logger.info("Crawler data " + " > " + modelTrain.getTrainNumber());
					}
				} catch (ServiceException e) {
					e.printStackTrace();
				}
			}
		} else if (isTrainListPage(url)) {
			logger.info("Crawl train list page " + " > " + url);
		} else {
			return;
		}
	}
	
	public Object getMyLocalData() {
		return null;
	}
	
	public void onBeforeExit() {
		
		/*
		 * log the page not be accessed
		 */
		if (errorPages != null && errorPages.size() > 0) {
			for (String errorPage : errorPages) {
				logger.error("Error pages found: " + errorPage);
			}
		}
		
		/*
		 * data validation with official site
		 */
		
	}
	
	/**
	 * 
	 * @param href
	 * @return
	 */
	private boolean isTrainPage (String href) {
		return href.indexOf("%") == -1 && href.toLowerCase().startsWith("http://www.tielu.org/search/"); 
	}
	
	/**
	 * 
	 * @param href
	 * @return
	 */
	private boolean isTrainListPage (String href) {
		return href.indexOf("%") == -1 && href.toLowerCase().startsWith("http://www.tielu.org/trainlist/trainlist-"); 
	}
	
	/**
	 * 
	 * @param href
	 * @return
	 */
	private String translateTrainNo (String href) {
		if (UtilsString.isNotEmpty(href) && href.toLowerCase().startsWith("http://www.tielu.org/search/")) {
			String linkNo = href.toLowerCase().substring("http://www.tielu.org/search/".length(), href.lastIndexOf("."));
			return linkNo.replaceAll("-", "/");
		}
		return null;
	}
	
	/**
	 * 
	 * @param text
	 * @return
	 */
	private String clearTags (String text) {
		if (UtilsString.isNotEmpty(text)) {
			text = text.replaceAll("<.*>", text);
		}
		return text;
	}
	
	/**
	 * 
	 * @param htmlContent
	 * @param contextURL
	 * @return 
	 */
	private ModelTrain getTrainInfo(String htmlContent, String contextURL) {
		if (UtilsString.isNotEmpty(htmlContent)) {
			Document htmlDoc = Jsoup.parse(htmlContent);
			Elements eleTables = htmlDoc.select("div.ResultTrainCodeContent > table");
			if (eleTables != null && eleTables.size() > 2) {
				
				/*
				 * ---------------------------------------
				 * 杞︽		G2008		杩愯鏃堕棿		2鏃�0鍒�
				 * 濮嬪彂绔�瑗垮畨鍖�	缁堢偣绔�	閮戝窞
				 * 鍙戣溅鏃堕棿 	10:10		鍒扮珯鏃堕棿		13:00
				 * 绫诲瀷 		楂橀�鍔ㄨ溅		鍏ㄧ▼ 		505鍏噷
				 * --------------------------------------
				 */
				
				ModelTrain model = new ModelTrain();
				
				boolean error = false;
				
				Element eleInfoTrain = eleTables.get(1);
				Elements eleInfo = eleInfoTrain.select("td[bgcolor=#f1f9fe]");
				if (eleInfo != null && eleInfo.size() > 0) {
					
					try {
						model.setTrainNumber(clearTags(eleInfo.get(0).text()));
						model.setTrainRuntime(clearTags(eleInfo.get(1).text()));
						model.setTrainStationStarting(clearTags(eleInfo.get(2).text()));
						model.setTrainStationTerminating(clearTags(eleInfo.get(3).text()));
						model.setTrainTimeDeparture(clearTags(eleInfo.get(4).text()));
						model.setTrainTimeArrival(clearTags(eleInfo.get(5).text()));
						model.setTrainType(clearTags(eleInfo.get(6).text()));
						model.setTrainRange(clearTags(eleInfo.get(7).text()));
					} catch (Exception e) {
						e.printStackTrace();
						error = true;
					}
				}
				
				/*
				 * 鍔ㄨ溅[D]
				 * ----------------------------------------------------------------------------------------------------------------
				 * No.	杞︾珯		鐏溅绁ㄤ氦娴佷氦娴�	閰掑簵 	鏃ユ湡 	鍋滆溅 鏃堕棿 	寮�溅 鏃堕棿 	閲岀▼ 	涓�瓑 杞骇 	浜岀瓑 杞骇		鐗圭瓑搴�
				 * ----------------------------------------------------------------------------------------------------------------
				 * 
				 * 楂橀搧[G]
				 * ----------------------------------------------------------------------------------------------------------------
				 * No.	杞︾珯		鐏溅绁ㄤ氦娴佷氦娴�	閰掑簵 	鏃ユ湡 	鍋滆溅 鏃堕棿 	寮�溅 鏃堕棿 	閲岀▼ 	涓�瓑 杞骇 	浜岀瓑 杞骇		鐗圭瓑搴�
				 * -----------------------------------------------------------------------------------------------------------------
				 * 
				 * 鏅揩
				 * -----------------------------------------------------------------------------------------------------------------
				 * No.	杞︾珯		鐏溅绁ㄤ氦娴佷氦娴�	閰掑簵 	鏃ユ湡 	鍋滆溅 鏃堕棿 	寮�溅 鏃堕棿 	閲岀▼ 	纭骇 	纭崸涓摵		杞骇		杞崸涓嬮摵
				 * -----------------------------------------------------------------------------------------------------------------
				 * 
				 */
				if (UtilsString.isNotEmpty(model.getTrainNumber())) {
					Element eleInfoSchedule = eleTables.get(2);
					Elements eleSchedules = eleInfoSchedule.select("tr[bgcolor=#f1f9fe]");
					if (eleSchedules != null && eleSchedules.size() > 0) {
						for (Element ele : eleSchedules) {
							ModelTrainSchedule modelSchedule = new ModelTrainSchedule();
							
							Elements schedule = ele.select("td");
							if (schedule != null) {
								try {
									modelSchedule.setTrainStationSeq(Integer.parseInt(clearTags(schedule.get(0).text())));
									modelSchedule.setTrainStationName(clearTags(schedule.get(1).text()));
									modelSchedule.setTrainArriveDate(clearTags(schedule.get(4).text()));
									modelSchedule.setTrainArriveTime(clearTags(schedule.get(5).text()));
									modelSchedule.setTrainDepartureTime(clearTags(schedule.get(6).text()));
									modelSchedule.setTrainRange(clearTags(schedule.get(7).text()));
									if (model.getTrainNumber().toUpperCase().startsWith("G") || 
											model.getTrainNumber().toUpperCase().startsWith("D")) {
										if (schedule.size() > 8) {
											modelSchedule.setTrainPriceSoftLv1st(
													Integer.parseInt(clearTags(schedule.get(8).text())));
										}
										if (schedule.size() > 9) {
											modelSchedule.setTrainPriceSoftLv2nd(
													Integer.parseInt(clearTags(schedule.get(9).text())));
										}
										if (schedule.size() > 10) {
											modelSchedule.setTrainPriceSoftLvBest(
													Integer.parseInt(clearTags(schedule.get(10).text())));
										}
									} else {
										if (schedule.size() > 8) {
											modelSchedule.setTrainPriceHardSeat(
													Integer.parseInt(clearTags(schedule.get(8).text())));
										}
										if (schedule.size() > 9) {
											modelSchedule.setTrainPriceHardBerthTop(
													Integer.parseInt(clearTags(schedule.get(9).text())));
										}
										if (schedule.size() > 10) {
											modelSchedule.setTrainPriceSoftSeat(
													Integer.parseInt(clearTags(schedule.get(10).text())));
										}
										if (schedule.size() > 11) {
											modelSchedule.setTrainPriceSoftBerthTop(
													Integer.parseInt(clearTags(schedule.get(11).text())));
										}
									}
									modelSchedule.setTrainModel(model);
								} catch (Exception e) {
									error = true;
									e.printStackTrace();
								}
								
								if (!error) {
									model.getTrainSchedules().add(modelSchedule);
								} else {
									System.out.println(">>>> Errors found," + contextURL);
									if (errorPages == null) {
										errorPages = new ArrayList<String>();
									}
									errorPages.add(contextURL);
								}
							}
						}
					}
				}
				return model;
			}
		}
		return null;
	}
	
	public static void main (String[] args) {
	}

	public static Logger getLogger() {
		return logger;
	}
	
}
